package cn.oldsix.spark.core.anagram

import org.apache.spark.{SparkConf, SparkContext}

/**
  * @ Author : Wu.D.J
  * @ Create : 2017.07.31
  */
object SameWordChoose {
    def main(args: Array[String]): Unit = {
        val conf = new SparkConf().setAppName("Anagram")
        val sc = new SparkContext(conf)
        val input = "/data/spark-example/anagram/anagram.txt"
        val output = "hdfs://oldsix1:9000/data/spark-example/anagram/output4/"
        val words = sc.textFile(input)
        val mapRDD = words.map(word => (word.toCharArray().sorted.mkString(""), word))
        val reduceRDD = mapRDD.reduceByKey((x, y) => x + "," + y)
        reduceRDD.sortBy(x => x).filter(x => x._2.split(",").length > 1).map(x => x._2).saveAsTextFile(output)
        sc.stop()
    }
}
